import { Atom } from './atom-class';

import {
  argAtoms,
  getDefinition,
  getEnvironmentDefinition,
} from '../latex-commands/definitions-utils';
import type { ColumnFormat } from '../atoms/array';

import { ErrorAtom } from '../atoms/error';
import { GroupAtom } from '../atoms/group';
import { LeftRightAtom } from '../atoms/leftright';
import { MacroAtom } from '../atoms/macro';
import { PromptAtom } from '../atoms/prompt';
import { PlaceholderAtom } from '../atoms/placeholder';
import { SubsupAtom } from '../atoms/subsup';
import { TextAtom } from '../atoms/text';

import { Mode } from './modes-utils';
import { joinLatex, tokenize, tokensToString } from './tokenizer';
import type {
  Style,
  ParseMode,
  LatexSyntaxError,
  Dimension,
  Glue,
  ArgumentType,
  Token,
  MathstyleName,
  Environment,
  LatexValue,
  DimensionUnit,
} from '../public/core-types';
import type {
  BBoxParameter,
  ContextInterface,
  PrivateStyle,
} from '../core/types';
import { Context } from './context';
import { Argument, LatexCommandDefinition } from 'latex-commands/types';
import { codePointToLatex } from './unicode';
import { isArray } from '../common/types';

//
// - Literal (character token): a letter, digit or punctuation
// - Token: a space `<space>`, a literal, name, group or mode shift
// - Name (control sequence): a token with an initial `\` followed by
//      one or more letters /[a-zA-Z]+\*?/ or followed by a single
//      non-letter (the name of some commands such as `operatorname*`
//      and `hspace*` names end with a `*`) or the `~` token, e.g.
//      `\frac`, `\alpha`, `\!`
// - Symbol: a name which is not a command, with no arguments,
//      e.g. `\pi`
// - Group: a sequence of tokens that start with `<{>` and end
//      with `<}>`
// - Argument: a single token or group after a name
// - Command:  a name followed by some optional argument, and one
//      or more required arguments
//

// Performance to check first char of string: https://jsben.ch/QLjdZ

// A literal is a token other than a special token. It
// may include a multi-character sequence, for example 🧑🏻‍🚀
function isLiteral(token: Token | undefined): boolean {
  if (!token) return false;
  return !/^(<$$>|<$>|<space>|<{>|<}>|#[0-9\?]|\\.+)$/.test(token);
}

// The `ParsingContext` is the set of properties that get 'reset' when a
// new context is entered, essentially within a `{...}` or `[...]`...
// Their values are restored when the context is exited.

export interface ParsingContext {
  parent: ParsingContext | undefined;

  // The `parse()` function for a command can modify:
  // - the `mathlist` to examine the context of the command, modify atoms
  // in the current mathlist being constructed (e.g. `\limits`) or add
  // to the mathlist (when creating a new Atom)
  // - the `style` for commands that modify the font style, color, etc...
  // - the `parseMode` for commands that shift into math or text mode
  // - the `mathstyle` for commands such as `\displaystyle`
  mathlist: Atom[];
  style: Style;

  // The parser keeps track of the state of these variable, as they can
  // influence the parsing, e.g. the `\mathchoice` command
  parseMode: ParseMode;
  mathstyle: MathstyleName;

  // When in tabular mode, `"&"` is interpreted as a column separator and
  // `"\\"` as a row separator. Used for matrixes, etc...
  tabular: boolean;
}

/**
 * Transform a list of tokens into a list of atoms (a mathlist in TeX's parlance)
 *
 */
export class Parser {
  // Accumulated errors encountered while parsing
  errors: LatexSyntaxError[] = [];

  // An array of tokens generated by the lexer
  private tokens: Token[];

  // The current token to be parsed: index in `this.tokens`
  private index = 0;

  // Substitutes for `#` tokens.
  private args?: (arg: string) => string | undefined;

  // Counter to prevent deadlock. If `end()` is called too many
  // times (1,000) in a row for the same token, bail.
  private endCount = 0;

  private parsingContext: ParsingContext;

  private context: Context;

  // smartFence  is also in Rootontext, but it needs to be
  // overridden in some cases
  private smartFence: boolean;

  /**
   * @param tokens - An array of tokens generated by the lexer.
   *
   * Note: smartFence and registers are usually defined by the GloablContext.
   * However, in some cases they need to be overridden.
   *
   */
  constructor(
    tokens: Token[],
    context?: ContextInterface,
    options?: {
      args?: (arg: string) => string | undefined;
      parseMode?: ParseMode;
      mathstyle?: MathstyleName;
      style?: Style;
    }
  ) {
    options ??= {};
    this.tokens = tokens;
    this.context =
      context instanceof Context && !options?.parseMode && !options.mathstyle
        ? context
        : new Context(
            { from: context, mathstyle: options.mathstyle },
            options.style
          );

    this.args = options.args ?? undefined;
    this.smartFence = this.context.smartFence;
    this.parsingContext = {
      parent: undefined,
      mathlist: [],
      style: options.style ?? {},
      parseMode: options.parseMode ?? 'math',
      mathstyle: options.mathstyle ?? 'displaystyle',
      tabular: false,
    };
  }

  beginContext(options?: {
    mode?: ParseMode;
    mathstyle?: MathstyleName;
    tabular?: boolean;
  }): void {
    const current = this.parsingContext;
    const newContext: ParsingContext = {
      parent: current,
      mathlist: [],
      style: { ...current.style },
      parseMode: options?.mode ?? current.parseMode,
      mathstyle: options?.mathstyle ?? current.mathstyle,
      tabular: options?.tabular ?? false,
    };
    this.parsingContext = newContext;
  }

  endContext(): void {
    this.parsingContext = this.parsingContext.parent!;
  }

  onError(err: LatexSyntaxError): void {
    this.errors.push({
      before: tokensToString(this.tokens.slice(this.index, this.index + 10)),
      after: tokensToString(
        this.tokens.slice(Math.max(0, this.index - 10), this.index)
      ),
      ...err,
    });
  }

  get mathlist(): Atom[] {
    return this.parsingContext.mathlist;
  }

  set mathlist(value: Atom[]) {
    this.parsingContext.mathlist = value;
  }

  get parseMode(): ParseMode {
    return this.parsingContext.parseMode;
  }

  set parseMode(value: ParseMode) {
    this.parsingContext.parseMode = value;
  }

  get tabularMode(): boolean {
    return this.parsingContext.tabular;
  }

  get style(): PrivateStyle {
    // Style is inherited
    let context: ParsingContext | undefined = this.parsingContext;
    while (context) {
      if (context.style) return { ...context.style };
      context = context.parent;
    }
    return {};
  }

  set style(value: Style) {
    this.parsingContext.style = value;
  }

  /**
   * True if we've reached the end of the token stream
   */
  end(): boolean {
    // To prevent a deadlock, count how many times end() is called without the
    // index advancing. If it happens more than 1,000 times in a row,
    // assume something is broken and pretend the stream is finished.
    this.endCount++;
    return this.index >= this.tokens.length || this.endCount > 1000;
  }

  next(): void {
    this.index += 1;
  }

  get(): Token {
    this.endCount = 0;
    return this.index < this.tokens.length ? this.tokens[this.index++] : '';
  }

  peek(): Token | undefined {
    return this.tokens[this.index];
  }

  // If the next token is a Unicode character such as ² or ℂ,
  // expand it with an equivalent LaTeX command.
  expandUnicode(): void {
    if (!this.peek()) return;

    if (this.parseMode !== 'math') return;

    // Check if we have a Unicode character such as `²` or `ℂ`
    const latex = codePointToLatex(this.peek()!);
    if (latex) this.tokens.splice(this.index, 1, ...tokenize(latex));
  }

  /**
   * @return True if the next token matches the input, and advance
   */
  match(input: string): boolean {
    if (this.tokens[this.index] === input) {
      this.index++;
      return true;
    }

    return false;
  }

  /**
   * Return the last atom in the mathlisst that can have a
   * subscript/superscript attached to it.
   * If there isn't one, insert a `SubsupAtom` and return it.
   */
  lastSubsupAtom(): Atom {
    let atom: Atom;
    if (this.mathlist.length > 0) {
      atom = this.mathlist[this.mathlist.length - 1];

      // If this is a `subsup` atom, it can have a `subsup` attached to it.
      if (atom.type === 'subsup') return atom;

      // An atom can have superscript/subscript attached to it if it accepts
      // limits (`\sum`, `\vec`...)
      if (atom.subsupPlacement !== undefined) return atom;
    }
    // Create a new `subsup` atom and return it
    atom = new SubsupAtom({ style: this.style });
    this.mathlist.push(atom);
    return atom;
  }

  /**
   * @return True if the next token matches the specified regular expression pattern.
   */
  hasPattern(pattern: RegExp): boolean {
    return pattern.test(this.tokens[this.index]);
  }

  hasInfixCommand(): boolean {
    const { index } = this;
    if (index < this.tokens.length && this.tokens[index].startsWith('\\')) {
      const info = getDefinition(this.tokens[index], this.parseMode);
      if (!info || info.definitionType === 'symbol') return false;
      if (info.ifMode && !info.ifMode.includes(this.parseMode)) return false;

      return info.infix ?? false;
    }

    return false;
  }

  matchColumnSeparator(): boolean {
    if (!this.tabularMode) return false;
    const peek = this.peek();
    if (peek !== '&') return false;
    this.index++;
    return true;
  }

  matchRowSeparator(): boolean {
    if (!this.tabularMode) return false;
    const peek = this.peek();
    if (peek !== '\\\\' && peek !== '\\cr' && peek !== '\\tabularnewline')
      return false;
    this.index++;
    return true;
  }

  /**
   * Return the appropriate value for a placeholder, either a default
   * one, or if a value was provided for #? via args, that value.
   */
  placeholder(): Atom[] {
    const placeHolderArg = this.args?.('?');
    if (!placeHolderArg)
      return [new PlaceholderAtom({ mode: this.parseMode, style: this.style })];

    // If there is a specific value defined for the placeholder,
    // use it.
    return parseLatex(placeHolderArg, {
      parseMode: this.parseMode,
      mathstyle: 'textstyle',
    });
  }

  skipWhitespace(): void {
    while (this.match('<space>')) {}
  }

  skipUntilToken(input: string): void {
    let token = this.tokens[this.index];
    while (token && token !== input) token = this.tokens[++this.index];

    if (token === input) this.index++;
  }

  skipFiller(): void {
    while (this.match('\\relax') || this.match('<space>')) {}
  }

  /**
   * Keywords are used to specify dimensions, and for various other
   * syntactic constructs.
   *
   * Unlike commands, they are not case sensitive.
   *
   * There are 25 keywords:
   *
   * at by bp cc cm dd depth em ex fil fill filll height in minus
   * mm mu pc plus pt sp spread to true width
   *
   * TeX: 8212
   * @return true if the expected keyword is present
   */
  matchKeyword(keyword: string): boolean {
    const savedIndex = this.index;
    let done = this.end();
    let value = '';
    while (!done) {
      const token = this.get();
      if (isLiteral(token)) {
        value += token;
        done = this.end() || value.length >= keyword.length;
      } else done = true;
    }

    const hasKeyword = keyword.toUpperCase() === value.toUpperCase();
    if (!hasKeyword) this.index = savedIndex;

    return hasKeyword;
  }

  /**
   * Return a sequence of characters as a string.
   * i.e. 'abcd' returns 'abcd'.
   * Terminates on the first non-literal token encountered
   * e.g. '<{>', '<}>' etc...
   * Will also terminate on character literal ']'
   */
  scanString(): string {
    let result = '';
    while (!this.end()) {
      const token = this.peek()!;

      if (token === ']') return result;

      if (token === '<space>') result += ' ';
      else if (token.startsWith('\\')) {
        // TeX will give a "Missing \endcsname inserted" error
        // if it encounters any command when expecting a string.
        // We're a bit more lax.
        this.onError({ code: 'unexpected-command-in-string' });
        result += token.substring(1);
      } else if (isLiteral(token)) result += token;
      else {
        // It's '<{>', '<}>', '<$>' or '<$$>
        return result;
      }
      this.next();
    }

    return result;
  }

  /**
   * Return a sequence of characters as a string.
   * Terminates on a balanced closing bracket
   * This is used by the `\ce` command
   */
  scanBalancedString(): string {
    let result = '';
    let done = this.end();
    let level = 1;
    while (!done) {
      const token = this.get();

      if (token === '<space>') result += ' ';
      else if (token === '<{>') {
        result += '{';
        level += 1;
      } else if (token === '<}>') {
        level -= 1;
        if (level > 0) result += '}';
        else this.index -= 1;
      } else if (token === '<$>') result += '$';
      else if (token === '<$$>') result += '$$';
      else result += token;

      done = level === 0 || this.end();
    }

    return result;
  }

  /**
   * Return the literal tokens, as a string, until a matching closing "}"
   * Used when handling macros
   */
  scanLiteralGroup(): string {
    if (!this.match('<{>')) return '';

    let result = '';
    let level = 1;
    while (level > 0 && !this.end()) {
      const token = this.get()!;
      if (token === '<}>') {
        level -= 1;
        // Don't include final '}'
        if (level > 0) result += '}';
      } else if (token === '<{>') {
        level += 1;
        result += '{';
      } else {
        if (/\\[a-zA-Z]+$/.test(result) && /^[a-zA-Z]/.test(token))
          result += ' ';
        result +=
          {
            '<space>': ' ',
            '<$$>': '$$',
            '<$>': '$',
          }[token] ?? token;
      }
    }

    return result;
  }

  /**
   * Return as a number a group of characters representing a
   * numerical quantity.
   *
   * From TeX:8695 (scan_int):
   * > An integer number can be preceded by any number of spaces and `+' or
   * > `-' signs. Then comes either a decimal constant (i.e., radix 10), an
   * > octal constant (i.e., radix 8, preceded by '), a hexadecimal constant
   * > (radix 16, preceded by "), an alphabetic constant (preceded by `), or
   * > an internal variable.
   */
  scanNumber(isInteger = true): null | {
    number: number;
    base?: 'decimal' | 'octal' | 'hexadecimal' | 'alpha';
  } {
    let negative = false;
    let token = this.peek();
    // TeXBook p.269.
    while (token === '<space>' || token === '+' || token === '-') {
      this.get();
      if (token === '-') negative = !negative;
      token = this.peek();
    }

    isInteger = Boolean(isInteger);

    let radix = 10;
    let digits = /\d/;
    if (this.match("'")) {
      // Apostrophe indicates an octal value
      radix = 8;
      digits = /[0-7]/;
      isInteger = true;
    } else if (this.match('"')) {
      // Double-quote indicates a hex value
      radix = 16;
      // Hex digits have to be upper-case
      digits = /[\dA-F]/;
      isInteger = true;
    } else if (this.match('x')) {
      // The 'x' prefix notation for the hexadecimal numbers is a MathJax extension.
      // For example: 'x3a'
      radix = 16;
      digits = /[\dA-Fa-f]/;
      isInteger = true;
    } else if (this.match('`')) {
      // A backtick indicates an alphabetic constant: a letter, or a single-letter command
      token = this.get();
      if (token) {
        if (token.length === 2 && token.startsWith('\\')) {
          return {
            number: (negative ? -1 : 1) * (token.codePointAt(1) ?? 0),
            base: 'alpha',
          };
        }
        return {
          number: (negative ? -1 : 1) * (token.codePointAt(0) ?? 0),
          base: 'alpha',
        };
      }

      return null;
    }

    let value = '';
    while (this.hasPattern(digits)) value += this.get();

    // Parse the fractional part, if applicable
    // Note: TeX does accept `,` as a decimal separator see TeX: `continental_point_token`
    if (!isInteger && (this.match('.') || this.match(','))) {
      value += '.';
      while (this.hasPattern(digits)) value += this.get();
    }

    const result: number = isInteger
      ? Number.parseInt(value, radix)
      : Number.parseFloat(value);
    if (Number.isNaN(result)) return null;
    return {
      number: negative ? -result : result,
      base: radix === 16 ? 'hexadecimal' : radix === 8 ? 'octal' : 'decimal',
    };
  }

  scanRegister(): LatexValue | null {
    const index = this.index;

    const number = this.scanNumber(false);

    this.skipWhitespace();
    if (this.match('\\relax')) return number;

    let negative = false;
    if (number === null) {
      while (true) {
        const s = this.peek();
        if (s === '-') negative = !negative;
        else if (s !== '+') break;
        this.next();
        this.skipWhitespace();
      }
    }
    if (this.match('\\global')) {
      this.skipWhitespace();
      const register = this.get();
      if (register.startsWith('\\')) {
        if (number) {
          return {
            register,
            global: true,
            factor: (negative ? -1 : 1) * number.number,
          };
        }
        if (negative) return { register, global: true, factor: -1 };
        return { register, global: true };
      }
      this.index = index;
      return null;
    }

    let register = this.get();
    if (!register?.startsWith('\\')) {
      this.index = index;
      return null;
    }

    register = register.substring(1);

    if (!this.context.registers[register]) {
      this.index = index;
      return null;
    }
    if (!negative || number !== null) {
      return {
        register,
        factor: (negative ? -1 : 1) * (number?.number ?? 1),
      };
    }
    return { register };
  }

  scanValue(): LatexValue | null {
    const register = this.scanRegister();
    if (register) return register;

    const index = this.index;

    const glue = this.scanGlueOrDimen();
    if (glue && ('unit' in glue || ('glue' in glue && 'unit' in glue.glue)))
      return glue;

    this.index = index;

    const number = this.scanNumber();
    if (number) return number;

    if (this.end() || !isLiteral(this.peek())) return null;

    const s = this.scanString();
    if (s.length > 0) return { string: s };

    return null;
  }

  /**
   * Return a dimension
   *
   * See TeX:8831
   */
  scanDimen(): Dimension | null {
    const value = this.scanNumber(false);
    if (value === null) return null;

    const dimension = value.number!;

    this.skipWhitespace();

    // The `true` keyword is used with magnification `\mag`
    // which we don't support. See TeXBook p. 270
    // > When ‘true’ is present, the factor is multiplied by 1000 and
    // > divided by the \mag parameter.
    this.matchKeyword('true');
    this.skipWhitespace();

    let unit: DimensionUnit | undefined;
    if (this.matchKeyword('pt')) unit = 'pt';
    else if (this.matchKeyword('mm')) unit = 'mm';
    else if (this.matchKeyword('cm')) unit = 'cm';
    else if (this.matchKeyword('ex')) unit = 'ex';
    else if (this.matchKeyword('px')) unit = 'px';
    else if (this.matchKeyword('em')) unit = 'em';
    else if (this.matchKeyword('bp')) unit = 'bp';
    else if (this.matchKeyword('dd')) unit = 'dd';
    else if (this.matchKeyword('pc')) unit = 'pc';
    else if (this.matchKeyword('in')) unit = 'in';
    else if (this.matchKeyword('mu')) unit = 'mu';

    return unit ? { dimension, unit } : { dimension };
  }

  scanGlueOrDimen(): Glue | Dimension | null {
    const dimen = this.scanDimen();
    if (dimen === null) return null;

    // After a dimension, a whitespace is consumed
    this.skipWhitespace();

    // `\\relax` can be used to indicate an end of value,
    // for example if there is the word "plus" right after, but which
    // should not be interpreted as part of the command
    // (note: we discard that \relax)
    if (this.match('\\relax')) return dimen;

    const result: Glue = { glue: dimen };
    // 'plus', optionally followed by 'minus'
    // ('minus' cannot come before 'plus')
    // dimen or 'hfill'
    if (this.matchKeyword('plus')) {
      // @todo there could also be a \hFilLlL command here
      const grow = this.scanDimen();
      if (grow) result.grow = grow;
      else return result;
    }

    this.skipWhitespace();
    if (this.match('\\relax')) return result;

    this.skipWhitespace();
    if (this.matchKeyword('minus')) {
      // @todo there could also be a \hFilLlL command here
      const shrink = this.scanDimen();
      if (shrink) result.shrink = shrink;
      else return result;
    }

    if (!result.grow && !result.shrink) return dimen;

    return result;
  }

  scanColspec(): ColumnFormat[] | null {
    this.skipWhitespace();
    const result: ColumnFormat[] = [];
    while (!this.end() && !(this.peek() === '<}>' || this.peek() === ']')) {
      const literal = this.get();
      if (literal === 'c' || literal === 'r' || literal === 'l')
        result.push({ align: literal });
      else if (literal === '|') result.push({ separator: 'solid' });
      else if (literal === ':') result.push({ separator: 'dashed' });
      else if (literal === '@') {
        if (this.match('<{>')) {
          this.beginContext({ mode: 'math' });
          result.push({
            gap: this.scan((token) => token === '<}>'),
          });
          this.endContext();
        }

        if (!this.match('<}>')) this.onError({ code: 'unbalanced-braces' });
      }
    }

    return result;
  }

  /**
   * Scan a `\(...\)` or `\[...\]` sequence
   * @return group for the sequence or null
   */
  scanModeSet(): Atom[] | null {
    let mathstyle: MathstyleName | undefined = undefined;
    if (this.match('\\(')) mathstyle = 'textstyle';
    if (!mathstyle && this.match('\\[')) mathstyle = 'displaystyle';
    if (!mathstyle) return null;
    this.beginContext({ mode: 'math', mathstyle });

    const result = this.scan(
      (token) => token === (mathstyle === 'displaystyle' ? '\\]' : '\\)')
    );

    if (!this.match(mathstyle === 'displaystyle' ? '\\]' : '\\)'))
      this.onError({ code: 'unbalanced-mode-shift' });

    this.endContext();

    return result;
  }

  /**
   * Scan a `$...$` or `$$...$$` sequence
   */
  scanModeShift(): Atom[] | null {
    let final: Token = '';
    if (this.match('<$>')) final = '<$>';
    if (!final && this.match('<$$>')) final = '<$$>';
    if (!final) return null;

    this.beginContext({
      mode: 'math',
      mathstyle: '<$>' ? 'textstyle' : 'displaystyle',
    });

    const result = this.scan((token) => token === final);

    if (!this.match(final)) this.onError({ code: 'unbalanced-mode-shift' });

    this.endContext();
    return result;
  }

  /**
   * Scan a \begin{env}...\end{end} sequence
   */
  scanEnvironment(): Atom | null {
    // An environment starts with a \begin command
    if (!this.match('\\begin')) return null;

    // The \begin command is immediately followed by the environment
    // name, as a string argument
    const envName = this.scanArgument('string') as Environment;
    if (!envName) return null;
    const def = getEnvironmentDefinition(envName);
    if (!def) {
      this.onError({
        code: 'unknown-environment',
        arg: envName,
      });
      return null;
    }

    // If the environment has some arguments, parse them
    const args: (null | Argument)[] = [];
    if (def.params) {
      for (const parameter of def.params) {
        // Parse an argument
        if (parameter.isOptional) {
          // If it's not present, parseOptionalArgument returns null,
          // but push it on the list of arguments anyway.
          // The null value will be interpreted as unspecified
          // optional value by the command parse function.
          args.push(this.scanOptionalArgument(parameter.type));
        } else {
          const arg = this.scanArgument(parameter.type);
          if (!arg) this.onError({ code: 'missing-argument', arg: envName });

          args.push(arg);
        }
      }
    }

    this.beginContext({ tabular: def.tabular });

    const array: Atom[][][] = [];
    const rowGaps: Dimension[] = [];
    let row: Atom[][] = [];
    let done = false;
    do {
      if (this.end()) {
        this.onError({ code: 'unbalanced-environment', arg: envName });
        done = true;
      }

      if (!done && this.match('\\end')) {
        if (this.scanArgument('string') !== envName) {
          this.onError({
            code: 'unbalanced-environment',
            arg: envName,
          });
        }

        done = true;
      }

      if (!done) {
        if (this.matchColumnSeparator()) {
          row.push(this.mathlist);
          this.mathlist = [];
        } else if (this.matchRowSeparator()) {
          row.push(this.mathlist);
          this.mathlist = [];
          let gap: Dimension | null = null;
          this.skipWhitespace();
          if (this.match('[')) {
            gap = this.scanDimen();
            this.skipWhitespace();
            this.match(']');
          }

          rowGaps.push(gap ?? { dimension: 0 });
          array.push(row);
          row = [];
        } else {
          this.mathlist.push(
            ...this.scan((token) =>
              [
                '<}>',
                '&',
                '\\end',
                '\\cr',
                '\\\\',
                '\\tabularnewline',
              ].includes(token)
            )
          );
        }
      }
    } while (!done);

    row.push(this.mathlist);
    if (row.length > 0) array.push(row);

    this.endContext();

    return def.createAtom(
      envName,
      array,
      rowGaps,
      args,
      this.context.maxMatrixCols
    );
  }

  /**
   * Parse an expression: a literal, or a command and its arguments
   */
  scanExpression(): Atom[] | null {
    const savedList = this.mathlist;
    this.mathlist = [];
    if (this.parseExpression()) {
      const result = this.mathlist;
      this.mathlist = savedList;
      return result;
    }
    this.mathlist = savedList;
    return null;
  }

  /**
   * Parse a sequence until a group end marker, such as
   * `}`, `\end`, `&`, etc...
   *
   * Returns an array of atoms or an empty array if the sequence
   * terminates right away.
   *
   * @param done - A predicate indicating if a token signals the end of a
   * group
   */
  scan(done?: (token: Token) => boolean): Atom[] {
    this.beginContext();

    // Default group end marker
    if (!done) done = (token: Token) => token === '<}>';

    // To handle infix commands, we'll keep track of their prefix
    // (tokens coming before them) and their arguments
    let infix: Token = '';
    let infixInfo: LatexCommandDefinition<[Atom[], Atom[]]> | null = null;
    let infixArgs: Atom[][] = [];
    let prefix: Atom[] | null = null;
    while (!this.end() && !done(this.peek()!)) {
      if (this.hasInfixCommand() && !infix) {
        // The next token is an infix and we have not seen one yet
        // (there can be only one infix command per implicit group).
        infix = this.get();
        // The current parseMode, this.parseMode, may no longer have the value
        // it had when we encountered the infix. However, since all infix are
        // only defined in 'math' mode, we can use the 'math' constant
        // for the parseMode
        infixInfo = getDefinition(infix, 'math') as LatexCommandDefinition<
          [Atom[], Atom[]]
        >;
        if (infixInfo) infixArgs = this.scanArguments(infixInfo)[1] as [Atom[]];

        // Save the math list so far and start a new one
        prefix = this.mathlist;
        this.mathlist = [];
      } else this.parseExpression();
    }

    let result: Atom[];
    if (infix) {
      console.assert(Boolean(infixInfo));
      infixArgs.unshift(this.mathlist); // Suffix
      if (prefix) infixArgs.unshift(prefix);
      result = [
        infixInfo!.createAtom!({
          command: infix,
          args: infixArgs as [Atom[], Atom[]],
          style: this.style,
          mode: this.parseMode,
        }),
      ];
    } else result = this.mathlist;

    this.endContext();

    return result;
  }

  /**
   * Parse a group enclosed in a pair of braces: `{...}`.
   *
   * Return either a group Atom or null if not a group.
   *
   * Return a group Atom with an empty body if an empty
   * group (i.e. `{}`).
   *
   * If the group only contains a placeholder, return a placeholder,
   */
  scanGroup(): Atom | null {
    const initialIndex = this.index;
    if (!this.match('<{>')) return null;

    const body = this.scan((token) => token === '<}>');
    if (!this.match('<}>')) this.onError({ code: 'unbalanced-braces' });

    if (body.length === 1 && body[0].type === 'placeholder') return body[0];

    const result = new GroupAtom(body, this.parseMode);
    result.verbatimLatex = tokensToString(
      this.tokens.slice(initialIndex, this.index)
    );

    return result;
  }

  scanSmartFence(): Atom | null {
    this.skipWhitespace();
    if (!this.match('(')) return null;
    // We've found an open paren... Convert to a `\left...\right`
    this.beginContext();
    let nestLevel = 1;
    while (!this.end() && nestLevel !== 0) {
      if (this.match('(')) nestLevel += 1;
      if (this.match(')')) nestLevel -= 1;
      if (nestLevel !== 0) this.parseExpression();
    }

    const result = new LeftRightAtom('', this.mathlist, {
      leftDelim: '(',
      rightDelim: nestLevel === 0 ? ')' : '?',
    });

    this.endContext();
    return result;
  }

  /**
   * Scan a delimiter, e.g. '(', '|', '\vert', '\ulcorner'
   *
   * @return The delimiter (as a character or command) or null
   */
  scanDelim(): string | null {
    this.skipWhitespace();
    const token = this.peek();
    if (!token) {
      this.onError({ code: 'unexpected-end-of-string' });
      return null;
    }

    if (!isLiteral(token) && !token.startsWith('\\')) return null;

    this.next();

    const info = getDefinition(token, 'math');
    if (!info) {
      this.onError({ code: 'unknown-command', arg: token });
      return null;
    }

    if (
      info.definitionType === 'function' &&
      info.ifMode &&
      !info.ifMode.includes(this.parseMode)
    ) {
      this.onError({ code: 'unexpected-delimiter', arg: token });
      return null;
    }

    if (
      info.definitionType === 'symbol' &&
      (info.type === 'mopen' || info.type === 'mclose')
    )
      return token;

    // Some symbols are not of type mopen/mclose, but are still
    // valid delimiters...
    // '?' is a special delimiter used as a 'placeholder'
    // (when the closing delimiter is displayed greyed out)
    if (
      /^(\.|\?|\||<|>|\\vert|\\Vert|\\\||\\surd|\\uparrow|\\downarrow|\\Uparrow|\\Downarrow|\\updownarrow|\\Updownarrow|\\mid|\\mvert|\\mVert)$/.test(
        token
      )
    )
      return token;

    this.onError({ code: 'unexpected-delimiter', arg: token });
    return null;
  }

  /**
   * Parse a `/left.../right` sequence.
   *
   * Note: the `/middle` command can occur multiple times inside a
   * `/left.../right` sequence, and is handled separately.
   *
   * Return either an atom of type `"leftright"` or null
   */
  scanLeftRight(): Atom | null {
    if (this.match('\\right')) {
      this.onError({ code: 'unbalanced-braces' });
      return new ErrorAtom('\\right');
    }
    if (this.match('\\mright')) {
      this.onError({ code: 'unbalanced-braces' });
      return new ErrorAtom('\\mright');
    }

    let close = '\\right';
    if (!this.match('\\left')) {
      if (!this.match('\\mleft')) return null;
      close = '\\mright';
    }

    const leftDelim = this.scanDelim();
    if (!leftDelim) {
      this.onError({ code: 'unexpected-delimiter' });
      return new ErrorAtom(close === '\\right' ? '\\left' : '\\mleft');
    }

    this.beginContext();

    while (!this.end() && !this.match(close)) this.parseExpression();

    const body = this.mathlist;
    this.endContext();

    // If we've reached the end and there was no `\right` or
    // there isn't a valid delimiter after `\right`, we'll
    // consider the `\right` missing and set the `rightDelim` to undefined
    const rightDelim = this.scanDelim() ?? '.';

    return new LeftRightAtom(
      close === '\\right' ? 'left...right' : 'mleft...mright',
      body,
      {
        leftDelim,
        rightDelim,
        style: this.style,
      }
    );
  }

  /**
   * Parse a subscript/superscript: `^` and `_`.
   *
   * Modify the last atom accordingly, or create a new 'subsup' carrier.
   *
   */
  parseSupSub(): boolean {
    // No sup/sub in text or command mode.
    if (this.parseMode !== 'math') return false;

    // Apply the subscript/superscript to the last rendered atom.
    // If none is present (beginning of the list, i.e. `{^2}`,
    // an empty atom will be created, equivalent to `{{}^2}`
    let token = this.peek();
    if (token !== '^' && token !== '_' && token !== "'") return false;

    const target = this.lastSubsupAtom();

    while (token === '^' || token === '_' || token === "'") {
      if (this.match("'")) {
        if (this.match("'")) {
          // A single quote, twice, is equivalent to '^{\doubleprime}'
          target.addChild(
            new Atom({
              type: 'mord',
              command: '\\doubleprime',
              mode: 'math',
              value: '\u2032\u2032', // "\u2033" displays too high
            }),
            'superscript'
          );
        } else {
          // A single quote (prime) is equivalent to '^{\prime}'
          target.addChild(
            new Atom({
              type: 'mord',
              command: '\\prime',
              mode: 'math',
              value: '\u2032',
            }),
            'superscript'
          );
        }
      } else if (this.match('^') || this.match('_')) {
        target.addChildren(
          argAtoms(this.scanArgument('expression')),
          token === '_' ? 'subscript' : 'superscript'
        );
      }

      token = this.peek();
    }

    return true;
  }

  /**
   * Parse a `\limits` or `\nolimits` command.
   *
   * This will change the placement of limits to be either above or below
   * (if `\limits`) or in the superscript/subscript position (if `\nolimits`).
   *
   * This overrides the calculation made for the placement, which is usually
   * dependent on the displaystyle (`textstyle` prefers `\nolimits`, while
   * `displaystyle` prefers `\limits`).
   */
  parseLimits(): boolean {
    // No limits in text or LaTeX mode.
    if (this.parseMode !== 'math') return false;

    // Note: `\limits`, `\nolimits` and `\displaylimits` are only applicable \
    // after an operator.
    // We skip them and ignore them if they are after something other
    // than an operator (TeX throws an error)

    const isLimits = this.match('\\limits');
    const isNoLimits = !isLimits && this.match('\\nolimits');
    const isDisplayLimits =
      !isNoLimits && !isLimits && this.match('\\displaylimits');

    if (!isLimits && !isNoLimits && !isDisplayLimits) return false;

    const opAtom =
      this.mathlist.length > 0 ? this.mathlist[this.mathlist.length - 1] : null;

    if (opAtom === null) return false;

    // Record that the limits was set through an explicit command
    // so we can generate the appropriate LaTeX later
    opAtom.explicitSubsupPlacement = true;

    if (isLimits) opAtom.subsupPlacement = 'over-under';
    if (isNoLimits) opAtom.subsupPlacement = 'adjacent';
    if (isDisplayLimits) opAtom.subsupPlacement = 'auto';

    return true;
  }

  scanArguments(
    info: Partial<LatexCommandDefinition>
  ): [ParseMode | undefined, (null | Argument)[]] {
    if (!info?.params) return [undefined, []];
    let deferredArg: ParseMode | undefined = undefined;
    const args: (null | Argument)[] = [];
    let i = info.infix ? 2 : 0;
    while (i < info.params.length) {
      const parameter = info.params[i];
      // Parse an argument
      if (parameter.type === 'rest') {
        args.push(
          this.scan((token) =>
            [
              '<}>',
              '&',
              '\\end',
              '\\cr',
              '\\\\',
              '\\tabularnewline',
              '\\right',
            ].includes(token)
          )
        );
      } else if (parameter.isOptional)
        args.push(this.scanOptionalArgument(parameter.type));
      else if (parameter.type.endsWith('*')) {
        // Indicate that a 'yet-to-be-parsed' argument is present
        // which should be accounted for *after* the command has been processed.
        deferredArg = parameter.type.slice(0, -1) as ParseMode;
      } else args.push(this.scanArgument(parameter.type));

      i += 1;
    }

    return [deferredArg, args];
  }

  /**
   * This function is similar to `scanSymbolOrCommand` but is is invoked
   * from a context where commands with arguments are not allowed, specifically
   * when parsing an unbraced argument, i.e. `\frac1\alpha`.
   *
   */
  scanSymbolOrLiteral(): Atom[] | null {
    const token = this.peek();
    if (!token) return null;
    this.next();
    let result: Atom | null;

    //
    // Is it a literal?
    //
    if (isLiteral(token)) {
      const result = Mode.createAtom(this.parseMode, token, { ...this.style });
      return result ? [result] : null;
    }

    //
    // Is this a macro?
    //
    result = this.scanMacro(token);
    if (result) return [result];

    //
    // Is this a command?
    //
    if (token.startsWith('\\')) {
      const def = getDefinition(token, this.parseMode);
      if (!def) {
        this.onError({ code: 'unknown-command', arg: token });
        return [new ErrorAtom(token)];
      }
      if (def.definitionType === 'symbol') {
        //
        // The command is a simple symbol (no arguments)
        //
        const style = { ...this.style };
        if (def.variant) style.variant = def.variant;

        result = new Atom({
          type: def.type,
          command: token,
          style,
          value: String.fromCodePoint(def.codepoint),
          mode: this.parseMode,
          verbatimLatex: token,
        });
      } else if (def.applyMode || def.applyStyle || def.infix) {
        // The command modifies the mode or style: can't use here
        this.onError({ code: 'invalid-command', arg: token });
        return [new ErrorAtom(token)];
      } else if (def.createAtom) {
        result = def.createAtom({
          command: token,
          args: [],
          style: this.style,
          mode: this.parseMode,
        });
      }
    }

    return result ? [result] : null;
  }

  /**
   * Parse a "math field" (in the TeX parlance), an argument to a
   * command.
   *
   * In LaTeX, the arguments of commands can either:
   * - require braces: e.g. `\hbox{\alpha}`
   * - prohibit braces: e.g. `\vskip 12pt`
   * - have optional braces, e.g. `\sqrt\frac12`. The behavior with and without
   * braces may be different. Example: `\not=` vs `\not{=}` render differently
   *
   * This is reflected by the argument type.
   * - Prohibited braces:
   *  - 'delim'
   * - Optional braces:
   *  - 'text': required braces (or when 'auto' means 'text')
   *  - 'math' (or when 'auto' means 'math')
   *  - 'expression'
   *  - 'value'
   * - Required braces:
   *  - 'string', 'balanced-string',
   *  - 'colspec'
   *  - 'bbox' (require square brackets)
   *
   * An argument can either be a single atom or  a sequence of atoms
   * enclosed in braces.
   *
   */
  scanArgument(type: 'auto'): null | Atom[] | { group: Atom[] };
  scanArgument(type: ParseMode): null | Atom[] | { group: Atom[] };
  scanArgument(type: 'expression'): null | Atom[] | { group: Atom[] };
  scanArgument(type: 'balanced-string'): null | string;
  scanArgument(type: 'colspec'): null | ColumnFormat[];
  scanArgument(type: 'delim'): null | string;
  scanArgument(type: 'value'): null | LatexValue;
  scanArgument(type: 'string'): null | string;
  scanArgument(type: ArgumentType): null | Argument;
  scanArgument(type: ArgumentType): null | Argument {
    this.skipFiller();
    const mode = this.parseMode;
    if (type === 'auto') type = mode;

    //
    // Argument without braces
    //
    if (!this.match('<{>')) {
      if (type === 'string') return this.scanString();
      if (type === 'value') return this.scanValue();
      if (type === 'delim') return this.scanDelim() ?? '.';
      if (type === 'expression') return this.scanExpression();
      if (type === 'math') {
        if (type !== mode) this.beginContext({ mode: 'math' });
        const result = this.scanSymbolOrLiteral();
        if (type !== mode) this.endContext();
        return result;
      }
      if (type === 'text') {
        if (type !== mode) this.beginContext({ mode: 'text' });
        const result = this.scanSymbolOrLiteral();
        if (type !== mode) this.endContext();
        return result;
      }
      if (type === 'balanced-string') return null;
      if (type === 'rest') {
        return this.scan((token) =>
          [
            '<}>',
            '&',
            '\\end',
            '\\cr',
            '\\\\',
            '\\tabularnewline',
            '\\right',
          ].includes(token)
        );
      }
      console.assert(false);
      return null;
    }

    //
    // Braced argument
    //

    if (type === 'text') {
      this.beginContext({ mode: 'text' });
      do this.mathlist.push(...this.scan());
      while (!this.match('<}>') && !this.end());
      const atoms = this.mathlist;
      this.endContext();
      // this.index -= 1;
      // const s = this.scanLiteralGroup();
      // const atoms = parseLatex(s, {
      //   context: this.context,
      //   parseMode: 'text',
      //   style: this.parsingContext.style,
      // });
      // this.endContext();
      return { group: atoms };
    }

    if (type === 'math') {
      this.beginContext({ mode: 'math' });
      do this.mathlist.push(...this.scan());
      while (!this.match('<}>') && !this.end());
      const atoms = this.mathlist;
      this.endContext();
      return { group: atoms };
    }

    let result: null | Argument = null;
    if (type === 'expression') {
      this.beginContext({ mode: 'math' });
      do this.mathlist.push(...this.scan());
      while (!this.match('<}>') && !this.end());
      const atoms = this.mathlist;
      this.endContext();
      return { group: atoms };
    }

    if (type === 'string') result = this.scanString();
    else if (type === 'balanced-string') result = this.scanBalancedString();
    else if (type === 'colspec') result = this.scanColspec();
    else if (type === 'value') result = this.scanValue();

    this.skipUntilToken('<}>');

    return result;
  }

  scanOptionalArgument(argType: ArgumentType): Argument | null {
    argType = argType === 'auto' ? this.parseMode : argType;
    this.skipFiller();
    if (!this.match('[')) return null;
    let result: Argument | null = null;
    while (!this.end() && !this.match(']')) {
      if (argType === 'string') result = this.scanString();
      else if (argType === 'value') result = this.scanValue();
      else if (argType === 'colspec') result = this.scanColspec();
      else if (argType === 'bbox') {
        // The \bbox command takes a very particular argument:
        // a comma delimited list of up to three arguments:
        // a color, a dimension and a string.
        // Split the string by comma delimited sub-strings, ignoring commas
        // that may be inside (). For example"x, rgb(a, b, c)" would return
        // ['x', 'rgb(a, b, c)']
        const bboxParameter: BBoxParameter = {};
        const list = this.scanString()
          .toLowerCase()
          .trim()
          .split(/,(?![^(]*\)(?:(?:[^(]*\)){2})*[^"]*$)/);
        for (const element of list) {
          const m = element.match(/^\s*([\d.]+)\s*([a-z]{2})/);
          if (m) {
            bboxParameter.padding = {
              dimension: parseInt(m[1]),
              unit: m[2] as DimensionUnit,
            };
          } else {
            const m = element.match(/^\s*border\s*:\s*(.*)/);
            if (m) bboxParameter.border = m[1];
            else bboxParameter.backgroundcolor = { string: element };
          }
        }

        result = bboxParameter;
      } else if (argType === 'math') {
        this.beginContext({ mode: 'math' });
        result = this.mathlist.concat(this.scan((token) => token === ']'));
        this.endContext();
      }
    }

    return result;
  }

  /** Parse a symbol or a command and its arguments
   * See also `scanSymbolOrLiteral` which is invoked from a context where
   * commands with arguments are not allowed, specifically when parsing an
   * unbraced argument, i.e. `\frac1\alpha`.
   */
  scanSymbolOrCommand(command: string): Readonly<Atom[]> | null {
    if (command === '\\placeholder') {
      const id = this.scanOptionalArgument('string') as string;
      // default value is legacy, ignored if there is a body
      // We need to check if second argument is `correct`, `incorrect` or to be interpreted as math
      const defaultValue = this.scanOptionalArgument('math') as Atom[];
      const defaultAsString = Atom.serialize(defaultValue, {
        defaultMode: 'math',
      });
      let defaultAtoms = [] as Atom[];

      let correctness;

      if (!correctness && defaultAsString === 'correct')
        correctness = 'correct';
      else if (!correctness && defaultAsString === 'incorrect')
        correctness = 'incorrect';
      else if (defaultAsString !== '') defaultAtoms = defaultValue;

      // const locked =  === 'locked';
      const locked = this.scanOptionalArgument('string') === 'locked';
      const value = this.scanArgument('auto');
      let body: Atom[];
      if (value && Array.isArray(value) && value.length > 0) body = value;
      else if (value && typeof value === 'object' && 'group' in value)
        body = value.group;
      else body = defaultAtoms;
      if (id) {
        return [
          new PromptAtom(id, correctness, locked, body ?? defaultAtoms, {
            mode: this.parseMode,
            style: this.style,
          }),
        ];
      }
      return [new PlaceholderAtom({ mode: this.parseMode, style: this.style })];
    }

    if (
      command === '\\renewcommand' ||
      command === '\\newcommand' ||
      command === '\\providecommand' ||
      command === '\\def'
    ) {
      // \\renewcommand: error if command is not already defined
      // \\newcommand: error if command is already defined
      // \\providecommand: define command only if it is not already defined
      // \\def: define command, silently overwriting any existing definition
      const index = this.index;
      // Get the command name
      const cmd = this.scanLiteralGroup() || this.next();
      if (!cmd) return null;

      // Define (or redefine) a command (or register)
      if (this.context.registers[cmd.substring(1)]) {
        const value = this.scanArgument('string');
        if (value !== null) this.context.registers[cmd.substring(1)] = value;

        const verbatimLatex = joinLatex([
          command,
          tokensToString(this.tokens.slice(index, this.index)),
        ]);

        return [new Atom({ type: 'text', value: '', verbatimLatex })];
      }

      // Could be a macro definition... @todo
      // \newcommand{\cmd}[nargs][optargdefault]{defn}
    }

    // Is this a macro?
    let result = this.scanMacro(command);
    if (result) return [result];

    // This wasn't a macro, so let's see if it's a regular command
    const info = getDefinition(command, this.parseMode);

    // An unknown command, or a command not available in this mode
    if (!info) {
      if (this.parseMode === 'text') {
        if (/[a-zA-Z]/.test(this.peek() ?? '')) {
          // The following character is a letter: insert a space
          // i.e. `\alpha x` -> `\alpha~x`
          // (the spaces are removed by the tokenizer)
          command += ' ';
        }
        return [...command].map(
          (c) =>
            new Atom({
              type: 'text',
              value: c,
              mode: 'text',
              style: this.style,
            })
        );
      }

      this.onError({ code: 'unknown-command', arg: command });
      return [new ErrorAtom(command)];
    }

    const initialIndex = this.index;

    if (info.definitionType === 'symbol') {
      const style = { ...this.style };

      // Override the variant if an explicit variant is provided
      if (info.variant) style.variant = info.variant;

      result = new Atom({
        type: info.type,
        command,
        style,
        value: String.fromCodePoint(info.codepoint),
        mode: this.parseMode,
      });
    } else {
      if (info.ifMode && !info.ifMode.includes(this.parseMode)) {
        // Command not applicable in this mode: ignore it (TeX behavior)
        return [];
      }

      // Parse the arguments.
      //
      // If `deferredArg` is not empty, the content after the command
      // will be parsed *after* the command has been initially processed
      // (atom creation or style application) and passed to
      //
      // This is used for commands such as \textcolor{color}{content}
      // that need to apply the color to the content *after* the
      // style has been changed.
      //
      // In definitions, this is indicated with a parameter type
      // thats ends with a '*' ('math*', 'auto*').

      const savedMode = this.parseMode;
      if (info.applyMode) this.parseMode = info.applyMode;

      let deferredArg: ParseMode | undefined = undefined;
      let args: (null | Argument)[] = [];
      if (info.parse) args = info.parse(this);
      else [deferredArg, args] = this.scanArguments(info);

      this.parseMode = savedMode;
      if (info.applyMode && !info.applyStyle && !info.createAtom)
        return argAtoms(args[0]);

      if (info.infix) {
        // Infix commands should be handled in scanImplicitGroup
        // If we find an infix command here, it's a syntax error
        // (second infix command in a group) and should be ignored.
        this.onError({
          code: 'too-many-infix-commands',
          arg: command,
        });
        return null;
      }

      //  Invoke the createAtom() function if present
      if (typeof info.createAtom === 'function') {
        result = info.createAtom({
          command,
          args: args as [],
          style: this.style,
          mode: this.parseMode,
        });
        if (deferredArg)
          result!.body = argAtoms(this.scanArgument(deferredArg));
      } else if (typeof info.applyStyle === 'function') {
        const style = info.applyStyle(this.style, command, args, this.context);

        // No type provided -> the parse function will modify
        // the current style rather than create a new Atom.
        const savedMode = this.parseMode;

        // Change to 'text' (or 'math') mode if necessary
        if (info.applyMode) this.parseMode = info.applyMode;

        // If a deferred arg is expected, process it now
        if (deferredArg) {
          // Create a temporary style
          const saveStyle = this.style;
          this.style = style;
          const atoms = this.scanArgument(deferredArg);
          this.style = saveStyle;
          this.parseMode = savedMode;
          return argAtoms(atoms);
        }

        // Merge the new style info with the current style
        this.style = style;
      } else {
        result = new Atom({
          type: 'mord',
          command: info.command ?? command,
          style: { ...this.style },
          value: command,
          mode: info.applyMode ?? this.parseMode,
        });
      }
    }

    if (!result) return null;

    if (
      result instanceof Atom &&
      result.verbatimLatex === undefined &&
      !/^\\(llap|rlap|class|cssId|htmlData)$/.test(command)
    ) {
      // We have to use `joinLatex` to correctly handle the case of
      // modal commands, e.g. `{\em m}`
      const verbatim = joinLatex([
        command,
        tokensToString(this.tokens.slice(initialIndex, this.index)),
      ]);
      if (verbatim) result.verbatimLatex = verbatim;
    }

    if (result.verbatimLatex === null) result.verbatimLatex = undefined;

    if (result.isFunction && this.smartFence) {
      // The command was a function that may be followed by
      // an argument, like `\sin(`
      const smartFence = this.scanSmartFence();
      if (smartFence) return [result, smartFence];
    }

    return [result];
  }

  scanSymbolCommandOrLiteral(): Readonly<Atom[]> | null {
    this.expandUnicode();

    const token = this.get();
    if (!token) return null;

    if (isLiteral(token)) {
      const result = Mode.createAtom(this.parseMode, token, { ...this.style });
      if (!result) return null;

      if (result.isFunction && this.smartFence) {
        // The atom was a function that may be followed by
        // an argument, like `f(`.
        const smartFence = this.scanSmartFence();
        if (smartFence) return [result, smartFence];
      }

      return [result];
    }

    if (token.startsWith('\\')) return this.scanSymbolOrCommand(token);

    if (token === '<space>') {
      if (this.parseMode === 'text')
        return [new TextAtom(' ', ' ', this.style)];

      return null;
    }

    if (token === '<}>') this.onError({ latex: '', code: 'unbalanced-braces' });
    else {
      this.onError({
        latex: '',
        code: 'unexpected-token',
        arg: token,
      });
    }

    return null;
  }

  /**
   * Scan the macro name and its arguments and return a macro atom
   */
  scanMacro(macro: string): Atom | null {
    const def = this.context.getMacro(macro);
    if (!def) return null;
    const initialIndex = this.index;
    const argCount = def.args!;

    // Carry forward the placeholder argument, if any.
    const args: Record<string, string | undefined> = { '?': this.args?.('?') };

    // Parse each argument group as a string. We don't know yet
    // what the proper parse mode is, so defer parsing till later
    // when invoking `parseLatex`
    for (let i = 1; i <= argCount; i++) {
      let arg = this.scanLiteralGroup();
      if (!arg) {
        // If the argument wasn't a group ({}), it may have
        // been a single token or expression, e.g. \frac12
        const index = this.index;
        this.scanExpression();
        arg = tokensToString(this.tokens.slice(index, this.index));
      }
      args[i] = arg;
    }

    // Group the result of the macro expansion
    return new MacroAtom(macro, {
      expand: def.expand,
      captureSelection: def.captureSelection,
      args:
        initialIndex === this.index
          ? null
          : tokensToString(this.tokens.slice(initialIndex, this.index)),
      style: this.parsingContext.style,
      body: parseLatex(def.def, {
        context: this.context,
        parseMode: this.parseMode,
        args: (arg) => args[arg],
        mathstyle: this.parsingContext.mathstyle,
        style: this.parsingContext.style,
      }),
    });
  }

  /**
   * Make an atom for the current token or token group and
   * add it to the parser's mathlist.
   * If the token is a command with arguments, will also parse the
   * arguments.
   */
  parseExpression(): boolean {
    let result: null | Atom | Readonly<Atom[]> =
      this.scanEnvironment() ??
      this.scanModeShift() ??
      this.scanModeSet() ??
      this.scanGroup() ??
      this.scanLeftRight();

    if (result === null) {
      if (this.parseSupSub()) return true;
      if (this.parseLimits()) return true;
      result = this.scanSymbolCommandOrLiteral();
    }

    // If we have an atom to add, push it at the end of the current
    // math list. We could have no atom for tokens that were skipped,
    // a ' ' in math mode for example
    if (!result) return false;
    if (isArray(result)) this.mathlist.push(...result);
    else this.mathlist.push(result);
    return true;
  }
}

/**
 * Given a string of LaTeX, return a corresponding array of atoms.
 * @param args - If there are any placeholder tokens, e.g.
 * `#0`, `#1`, etc... they will be replaced by the value provided by `args`.
 * @param smartFence - If true, promote plain fences, e.g. `(`,
 * as `\left...\right` or `\mleft...\mright`
 */
export function parseLatex(
  s: string,
  options?: {
    context?: ContextInterface;
    parseMode?: ParseMode;
    args?: (arg: string) => string | undefined;
    mathstyle?: MathstyleName;
    style?: Style;
  }
): Atom[] {
  const args = options?.args ?? undefined;
  const parser = new Parser(tokenize(s, args), options?.context, {
    args,
    mathstyle: options?.mathstyle ?? 'displaystyle',
    parseMode: options?.parseMode ?? 'math',
    style: options?.style ?? {},
  });

  const atoms: Atom[] = [];
  while (!parser.end()) atoms.push(...parser.scan(() => false));
  return atoms;
}

export function validateLatex(
  s: string,
  options?: { context: ContextInterface; parseMode?: ParseMode }
): LatexSyntaxError[] {
  const parser = new Parser(tokenize(s, null), options?.context, {
    mathstyle: 'displaystyle',
    parseMode: options?.parseMode ?? 'math',
  });

  while (!parser.end()) parser.scan();
  return parser.errors;
}
